﻿using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;

namespace MmCgDownloader
{
    class Program
    {
        class FailedInfo
        {
            public string Url;
            public HttpStatusCode? StatusCode;
        }

        static void Main(string[] args) {
            ServicePointManager.DefaultConnectionLimit = 10;

            // http://18av.mm-cg.com/cg_1.html
            for (int pageIndex = 1; pageIndex <= 9707; ++pageIndex) {
                try {
                    string pageUrl = string.Format("http://18av.mm-cg.com/cg_{0}.html", pageIndex);
                    PageInfo pageInfo = getPageInfo(pageUrl);
                    if (null != pageInfo) {
                        List<FailedInfo> failedInfos = new List<FailedInfo>();

                        // 出力先ディレクトリ作成
                        string title = pageInfo.Title;
                        char[] invalidChar = Path.GetInvalidFileNameChars();
                        foreach (char c in invalidChar) {
                            title = title.Replace(c, '_');
                        }
                        string savePageDir = "d:\\画像\\mm-cg\\" + pageIndex + " - " + title;
                        Directory.CreateDirectory(savePageDir);

                        // info.txtを出力
                        createInfoFile(pageUrl, pageInfo, savePageDir);
                        // ショートカットファイルを作成
                        createInternetShortcut(pageUrl, savePageDir);

                        Parallel.ForEach(pageInfo.ImageUrl, new ParallelOptions() { MaxDegreeOfParallelism = 3 }, (imageUrl, state, index) => {
                            try {
                                downloadImage(imageUrl, (int)index + 1, savePageDir);
                            }
                            catch (Exception exp) {
                                FailedInfo failedInfo = new FailedInfo() { Url = imageUrl };

                                WebException we = exp as WebException;
                                if (null != we && we.Status == WebExceptionStatus.ProtocolError)
                                    failedInfo.StatusCode = ((HttpWebResponse)we.Response).StatusCode;

                                Console.WriteLine(imageUrl + " のダウンロードに失敗");
                                lock (failedInfos) {
                                    failedInfos.Add(failedInfo);
                                }
                            }
                        });

                        // ダウンロードに失敗したファイルを failed～.txt に出力
                        if (0 < failedInfos.Count) {
                            DateTime now = DateTime.Now;
                            string failedFilePath = Path.Combine(savePageDir, string.Format("_failed_{0}.txt", now.ToString("yyMMddHH_mmss")));
                            using (StreamWriter writer = new StreamWriter(failedFilePath, true, _enc)) {
                                foreach (FailedInfo failedInfo in failedInfos) {
                                    writer.WriteLine(failedInfo.Url + "\t" + failedInfo.StatusCode);
                                }
                            }
                        }
                    }
                }
                catch (Exception exp) {
                    Console.WriteLine(exp.ToString());
                    break;
                }
            }
            Console.WriteLine("終了");
            Console.ReadKey();
        }

        static Encoding _enc = Encoding.GetEncoding("utf-8");

        static void createInfoFile(string pageUrl, PageInfo pageInfo, string savePageDir) {
            string infoFilePath = Path.Combine(savePageDir, "_info.txt");
            using (StreamWriter writer = new StreamWriter(infoFilePath, false, _enc)) {
                writer.WriteLine(pageUrl);
                writer.WriteLine(pageInfo.Title);
                writer.WriteLine("");
                foreach (string imageUrl in pageInfo.ImageUrl) {
                    writer.WriteLine(imageUrl);
                }
            }
        }

        static void createInternetShortcut(string targetUrl, string dir) {
            string linkFilePath = Path.Combine(dir, "_entry.url");
            Encoding enc = Encoding.GetEncoding("utf-8");
            using (StreamWriter sw = new StreamWriter(linkFilePath, false, enc)) {
                sw.WriteLine("[InternetShortcut]");
                sw.WriteLine("URL=" + targetUrl);
            }
        }

        static void downloadImage(string url, int imageIndex, string dir) {
            // 保存ファイルパスを作成
            string ext = Path.GetExtension(url);
            if (string.IsNullOrEmpty(ext))
                throw new Exception("urlから拡張子を取得するのに失敗");
            string filePath = Path.Combine(dir, string.Format("{0}{1}", imageIndex, ext));

            // 存在している場合、何もしない
            if (File.Exists(filePath))
                return;

            //サーバーからの応答を受信するためのWebResponseを取得
            WebResponse webres = getWebResponse(url);

            // メモリに格納
            MemoryStream ms = new MemoryStream();

                //応答データを受信するためのStreamを取得
            using (Stream rs = webres.GetResponseStream()) {
                byte[] buffer = new byte[65535];
                while (true) {
                    int rb = rs.Read(buffer, 0, buffer.Length);
                    if (rb > 0) {
                        ms.Write(buffer, 0, rb);
                    }
                    else {
                        break;
                    }
                }
            }

            // ファイルに保存
            FileStream fs = new FileStream(filePath, FileMode.Create);
            fs.Write(ms.GetBuffer(), 0, (int)ms.Length);
            fs.Close();
        }

        class PageInfo
        {
            public string Title;
            public List<string> ImageUrl = new List<string>();
        }

        static PageInfo getPageInfo(string url) {
            Console.WriteLine("★" + url);

            WebResponse res = getWebResponse(url);

            string src;

            using(Stream st = res.GetResponseStream())
            using (StreamReader sr = new StreamReader(st, _enc)) {
                src = sr.ReadToEnd();
            }

            //Trace.Write(src);

            PageInfo pageInfo = new PageInfo();

            // タイトルを取得
            Regex regex = new System.Text.RegularExpressions.Regex(
                    @"<title>(?<title>.*?)</title>",
                    RegexOptions.IgnoreCase | RegexOptions.Singleline);

            Match match = regex.Match(src);
            if (match.Success) {
                string title = match.Groups["title"].Value;
                title = title.Replace("\r", "");
                title = title.Replace("\n", "");
                title.Trim();
                pageInfo.Title = title;
            }
            if (null == pageInfo.Title)
                throw new Exception("タイトルの取得に失敗");

            // 画像のurlを取得
            // Large_cgurl[1] = "http://107.imagebam.com/download/oY_1RPjnoeGJxiSjwBmBfw/24091/240906225/41934_001.jpg";
            regex = new System.Text.RegularExpressions.Regex(
                    @"Large_cgurl\[\d+\][\x20\t]*=[\x20\t]*""(?<url>.*?)""",
                    System.Text.RegularExpressions.RegexOptions.IgnoreCase);

            for (match = regex.Match(src); match.Success; match = match.NextMatch()) {
                string imageUrl = match.Groups["url"].Value;
                pageInfo.ImageUrl.Add(imageUrl);
            }

            if (pageInfo.ImageUrl.Count <= 0)
                throw new Exception("画像URLの取得に失敗");

            return pageInfo;
        }

        static WebResponse getWebResponse(string url) {
            WebResponse res = null;
            int retry = 0;
            do {
                try {
                    WebRequest req = WebRequest.Create(url);
                    req.Timeout = 1000 * 60;
                    res = req.GetResponse();
                }
                catch (WebException we) {
                    // とりあえず404以外の場合、6回繰り返す。
                    if (retry++ >= 6 ||
                        (we.Status == WebExceptionStatus.ProtocolError && ((HttpWebResponse)we.Response).StatusCode == HttpStatusCode.NotFound)) {
                        throw we;
                    }
                    Console.WriteLine(DateTime.Now.ToString());
                    Console.WriteLine(we.ToString());
                    Thread.Sleep(TimeSpan.FromSeconds(10));
                    continue;
                }
            } while (null == res);
            return res;
        }

    }
}
